const { customRequire } = require('./utils');
const path = require('path');
const fs = require('fs');
const https = require('https');
const http = require('http');

// 使用 customRequire 加载第三方模块，解决打包后的模块路径问题
const { chromium } = customRequire('playwright');
const pdf = customRequire('pdf-parse');
const axiosModule = customRequire('axios');
const axios = axiosModule.default || axiosModule;
// const { JSDOM } = customRequire('jsdom'); // 如果需要使用 jsdom，取消注释此行
const pathUtils = require('../utils/path-utils');
const { formatDateToEnglish } = require('../utils/dateFormatter');
const { cleanupModuleData } = require('../utils');

// 目标链接映射
const TARGET_LINKS = {
    'Construction products (CPD/CPR)': '/single-market/goods/european-standards/harmonised-standards/construction-products-cpdcpr_en',
    'General product safety': '/single-market/goods/european-standards/harmonised-standards/general-product-safety_en',
    'Personal protective equipment (PPE)': '/single-market/goods/european-standards/harmonised-standards/personal-protective-equipment_en',
    'Toy safety': '/single-market/goods/european-standards/harmonised-standards/toy-safety_en',
    'Low Voltage (LVD)': '/single-market/goods/european-standards/harmonised-standards/low-voltage-lvd_en',
    'Electromagnetic compatibility (EMC)': '/single-market/goods/european-standards/harmonised-standards/electromagnetic-compatibility-emc_en',
    'Radio Equipment (RED)': '/single-market/goods/european-standards/harmonised-standards/radio-equipment_en',
    'Medical devices (Regulation 2017/745)': '/single-market/goods/european-standards/harmonised-standards/medical-devices_en',
    'In vitro diagnostic medical devices (Regulation 2017/746)': '/single-market/goods/european-standards/harmonised-standards/iv-diagnostic-medical-devices_en',
    'Gas appliances (GAR)': '/single-market/goods/european-standards/harmonised-standards/gas-appliances_en',
    'Machinery (MD)': '/single-market/goods/european-standards/harmonised-standards/machinery-md_en',
    'Recreational craft and personal watercraft': '/single-market/goods/european-standards/harmonised-standards/recreational-craft_en',
    'Pressure equipment (PED)': '/single-market/goods/european-standards/harmonised-standards/pressure-equipment_en',
    'Packaging and packaging waste': '/single-market/goods/european-standards/harmonised-standards/packaging-and-packaging-waste_en',
    'Plastic caps and lids': '/single-market/goods/european-standards/harmonised-standards/plastic-caps-and-lids_en'
};

const BASE_URL = 'https://single-market-economy.ec.europa.eu';
const MAIN_URL = 'https://single-market-economy.ec.europa.eu/single-market/goods/european-standards/harmonised-standards_en';

class EuCommissionScraper {
    constructor() {
        this.browser = null;
        this.context = null;
        this.page = null;
        this.results = [];
        this.isRunning = false;
        this.signal = null; // 添加信号支持
        this.fileSaved = false; // 添加文件保存标志，防止重复保存
    }

    // 调用智谱AI免费模型进行文本总结
    async generateSummary(text) {
        try {
            // 清理和截取文本，确保不超过API限制
            const cleanText = text.replace(/\s+/g, ' ').trim();
            const maxLength = 2000; // 限制输入文本长度
            const inputText = cleanText.length > maxLength ? cleanText.substring(0, maxLength) + '...' : cleanText;
            
            if (!inputText || inputText.length < 50) {
                return 'Text content is too short to generate summary';
            }

            // 使用智谱AI的免费API进行文本总结
            const response = await axios.post(
                'https://open.bigmodel.cn/api/paas/v4/chat/completions',
                {
                    model: 'glm-4-flash',
                    messages: [
                        {
                            role: 'system',
                            content: 'You are a professional text summarization assistant. Please generate a concise English summary for the text provided by the user, keeping it within 100 words and highlighting key information and main points.'
                        },
                        {
                            role: 'user',
                            content: `Please generate a concise English summary within 100 words for the following text:\n\n${inputText}`
                        }
                    ],
                    max_tokens: 150,
                    temperature: 0.1
                },
                {
                    headers: {
                        'Authorization': '5ede0a4acdf64ac3b0eb536fecb1f802.jzSUshgBIuc2K1oz', // 需要替换为实际的智谱AI API密钥
                        'Content-Type': 'application/json'
                    },
                    timeout: 15000 // 15秒超时
                }
            );

            if (response.data && response.data.choices && response.data.choices[0] && response.data.choices[0].message) {
                const summary = response.data.choices[0].message.content.trim();
                return summary || '无法生成摘要';
            } else {
                return '无法生成摘要';
            }
        } catch (error) {
            console.error('智谱AI总结生成失败:', error.message);
            // 如果AI总结失败，返回文本的前100个字符作为备用
            const fallbackSummary = text.replace(/\s+/g, ' ').trim().substring(0, 100);
            return fallbackSummary ? fallbackSummary + '...' : '无法生成摘要';
        }
    }

    // 初始化浏览器
    async init(headless = true, signal = null) {
        try {
            // 检查信号是否已经被中止
            if (signal && signal.aborted) {
                console.log('任务已被取消，不启动浏览器');
                throw new Error('任务已被用户取消');
            }
            
            this.signal = signal; // 保存信号引用
            
            this.browser = await chromium.launch({
                headless: headless,
                args: ['--no-sandbox', '--disable-setuid-sandbox']
            });
            
            // 如果信号存在，添加中断处理
            if (signal) {
                signal.addEventListener('abort', async () => {
                    console.log('收到中断信号，正在关闭浏览器...');
                    if (this.browser) {
                        try {
                            await this.browser.close();
                            console.log('浏览器已关闭（由中断信号触发）');
                        } catch (err) {
                            console.error('关闭浏览器时出错:', err);
                        }
                        this.browser = null;
                    }
                });
            }
            
            // 创建浏览器上下文，设置用户代理
            this.context = await this.browser.newContext({
                viewport: null,
                userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            });
            
            this.page = await this.context.newPage();
            
            console.log('EU Commission浏览器初始化成功');
            return true;
        } catch (error) {
            console.error('浏览器初始化失败:', error);
            return false;
        }
    }

    // 抓取目标页面
    async scrapeTargetPages(progressCallback) {
        if (!this.browser || !this.context) {
            throw new Error('浏览器未初始化');
        }

        this.isRunning = true;
        this.results = [];
        
        try {
            const targetEntries = Object.entries(TARGET_LINKS);
            const totalTargets = targetEntries.length;
            
            console.log(`开始处理 ${totalTargets} 个目标页面...`);
            
            let shouldStop = false;
            
            for (let i = 0; i < totalTargets; i++) {
                // 检查中断信号
                if (this.signal && this.signal.aborted) {
                    console.log('检测到中断信号，停止处理目标页面');
                    shouldStop = true;
                    break;
                }
                if (!this.isRunning) {
                    shouldStop = true;
                    break;
                }
                
                const [categoryName, linkPath] = targetEntries[i];
                const fullUrl = BASE_URL + linkPath;
                
                try {
                    console.log(`\n正在处理 [${i + 1}/${totalTargets}]: ${categoryName}`);
                    
                    // 更新进度
                    if (progressCallback) {
                        const progress = Math.round((i / totalTargets) * 100);
                        progressCallback(progress, `正在处理: ${categoryName}`);
                    }
                    
                    // 在新标签页中打开链接
                    const categoryPage = await this.context.newPage();
                    
                    // 增加超时时间并添加重试机制
                    let retryCount = 0;
                    const maxRetries = 3;
                    while (retryCount < maxRetries) {
                        try {
                            await categoryPage.goto(fullUrl, { waitUntil: 'networkidle', timeout: 60000 });
                            break; // 成功则跳出重试循环
                        } catch (error) {
                            retryCount++;
                            console.log(`   ⚠️ 页面加载失败，重试 ${retryCount}/${maxRetries}: ${error.message}`);
                            if (retryCount >= maxRetries) {
                                throw error; // 重试次数用完，抛出错误
                            }
                            await categoryPage.waitForTimeout(2000); // 等待2秒后重试
                        }
                    }
                    
                    // 等待页面完全加载
                    await categoryPage.waitForTimeout(2000);
                    
                    // 查找"Publications in the Official Journal"的h3标签（带或不带冒号）
                    const publicationsH3Elements = await categoryPage.$$('h3');
                    let foundPublicationsH3 = false;
                    
                    for (const h3 of publicationsH3Elements) {
                        const text = await h3.textContent();
                        if (text && text.includes('Publications in the Official Journal')) {
                            foundPublicationsH3 = true;
                            break;
                        }
                    }
                    
                    if (!foundPublicationsH3) {
                        console.log(`   ⚠️ 未找到"Publications in the Official Journal"标题`);
                        await categoryPage.close();
                        continue;
                    }
                    
                    // 获取该h3标签到下一个h3标签之间的所有链接
                    const links = await categoryPage.evaluate(() => {
                        const h3Elements = Array.from(document.querySelectorAll('h3'));
                        const publicationsH3 = h3Elements.find(h3 => h3.textContent.includes('Publications in the Official Journal'));
                        
                        if (!publicationsH3) return [];
                        
                        const nextH3Index = h3Elements.indexOf(publicationsH3) + 1;
                        const nextH3 = nextH3Index < h3Elements.length ? h3Elements[nextH3Index] : null;
                        
                        const links = [];
                        let currentElement = publicationsH3.nextElementSibling;
                        
                        while (currentElement && currentElement !== nextH3) {
                            const anchors = currentElement.querySelectorAll('a[href]');
                            anchors.forEach(anchor => {
                                const href = anchor.getAttribute('href');
                                const text = anchor.textContent.trim();
                                
                                // 排除RSS链接
                                if (href && text && !text.toLowerCase().includes('rss')) {
                                    links.push({
                                        url: href.startsWith('http') ? href : `https://eur-lex.europa.eu${href}`,
                                        text: text
                                    });
                                }
                            });
                            currentElement = currentElement.nextElementSibling;
                        }
                        
                        return links;
                    });
                    
                    console.log(`   📋 找到 ${links.length} 个链接`);
                    
                    // 处理每个链接
                    for (let j = 0; j < links.length; j++) {
                        // 检查中断信号
                        if (this.signal && this.signal.aborted) {
                            console.log('   检测到中断信号，中断链接处理');
                            shouldStop = true;
                            break;
                        }
                        if (!this.isRunning) {
                            console.log('   检测到停止信号，中断链接处理');
                            shouldStop = true;
                            break;
                        }
                        
                        const link = links[j];
                        console.log(`   正在处理链接 [${j + 1}/${links.length}]: ${link.text}`);
                        
                        try {
                            // 检查是否为PDF链接，如果是则直接处理，不使用page.goto
                            if (link.url.includes('PDF')) {
                                console.log(`   📄 检测到PDF链接，直接处理...`);
                                const jsonData = await this.extractJsonData(null, link);
                                
                                if (jsonData) {
                                    // PDF链接通常没有fullTextContent，但可以尝试从standardName生成简短总结
                                    if (jsonData.standardName && jsonData.standardName.trim().length > 50) {
                                        console.log(`   🤖 正在为PDF生成AI总结...`);
                                        try {
                                            const summary = await this.generateSummary(jsonData.standardName);
                                            jsonData.brief = summary;
                                            console.log(`   ✅ PDF AI总结生成成功: ${summary.substring(0, 50)}...`);
                                        } catch (summaryError) {
                                            console.log(`   ⚠️ PDF AI总结生成失败: ${summaryError.message}`);
                                            jsonData.brief = 'PDF总结生成失败';
                                        }
                                    }
                                    
                                    this.results.push(jsonData);
                                    console.log(`   ✅ 成功提取数据:`, JSON.stringify(jsonData, null, 2));
                                } else {
                                    console.log(`   ⚠️ 未能提取到有效数据`);
                                }
                                continue;
                            }
                            
                            // 非PDF链接的处理逻辑
                            // 在新标签页中打开链接
                            const detailPage = await this.context.newPage();
                            
                            // 检查中断信号
                            if (this.signal && this.signal.aborted) {
                                console.log('   检测到中断信号，中断页面加载');
                                await detailPage.close();
                                break;
                            }
                            // 检查停止信号
                            if (!this.isRunning) {
                                console.log('   检测到停止信号，中断页面加载');
                                await detailPage.close();
                                break;
                            }
                            
                            // 增加超时时间并添加重试机制
                            let linkRetryCount = 0;
                            const maxLinkRetries = 3;
                            while (linkRetryCount < maxLinkRetries) {
                                try {
                                    await detailPage.goto(link.url, { waitUntil: 'networkidle', timeout: 60000 });
                                    break; // 成功则跳出重试循环
                                } catch (error) {
                                    linkRetryCount++;
                                    console.log(`   ⚠️ 链接加载失败，重试 ${linkRetryCount}/${maxLinkRetries}: ${error.message}`);
                                    if (linkRetryCount >= maxLinkRetries) {
                                        throw error; // 重试次数用完，抛出错误
                                    }
                                    await detailPage.waitForTimeout(3000); // 等待3秒后重试
                                }
                            }
                            await detailPage.waitForTimeout(2000);
                            
                            // 再次检查中断信号
                            if (this.signal && this.signal.aborted) {
                                console.log('   检测到中断信号，中断数据提取');
                                await detailPage.close();
                                shouldStop = true;
                                break;
                            }
                            if (!this.isRunning) {
                                console.log('   检测到停止信号，中断数据提取');
                                await detailPage.close();
                                shouldStop = true;
                                break;
                            }
                            
                            // 检查并点击cookie接受按钮
                            try {
                                const cookieButton = detailPage.locator('a[href="#accept"].wt-ecl-button.wt-ecl-button--primary.wt-cck--actions-button');
                                if (await cookieButton.isVisible({ timeout: 3000 })) {
                                    await cookieButton.click();
                                    console.log(`   🍪 已点击cookie接受按钮`);
                                    await detailPage.waitForTimeout(1000);
                                }
                            } catch (cookieError) {
                                // 忽略cookie按钮相关错误，继续处理
                                console.log(`   ℹ️ 未找到cookie按钮或点击失败，继续处理`);
                            }
                            // }
                            
                            // 提取JSON数据
                            const jsonData = await this.extractJsonData(detailPage, link);
                            
                            // 检查中断信号
                            if (this.signal && this.signal.aborted) {
                                console.log('   检测到中断信号，中断数据处理');
                                await detailPage.close();
                                shouldStop = true;
                                break;
                            }
                            if (!this.isRunning) {
                                console.log('   检测到停止信号，中断数据处理');
                                await detailPage.close();
                                shouldStop = true;
                                break;
                            }
                            
                            if (jsonData) {
                                // 如果有fullTextContent，调用AI总结
                                if (jsonData.fullTextContent && jsonData.fullTextContent.trim().length > 0) {
                                    console.log(`   🤖 正在生成AI总结...`);
                                    try {
                                        const summary = await this.generateSummary(jsonData.fullTextContent);
                                        jsonData.brief = summary;
                                        console.log(`   ✅ AI总结生成成功: ${summary.substring(0, 50)}...`);
                                    } catch (summaryError) {
                                        console.log(`   ⚠️ AI总结生成失败: ${summaryError.message}`);
                                        jsonData.brief = '总结生成失败';
                                    }
                                }
                                
                                // 删除临时字段
                                delete jsonData.fullTextContent;
                                
                                this.results.push(jsonData);
                                console.log(`   ✅ 成功提取数据:`, JSON.stringify(jsonData, null, 2));
                            } else {
                                console.log(`   ⚠️ 未能提取到有效数据`);
                            }
                            
                            // 关闭详情页面
                            await detailPage.close();
                            
                            // 添加延迟避免请求过快
                            await categoryPage.waitForTimeout(1000);
                            
                        } catch (error) {
                            console.error(`   ❌ 处理链接失败: ${link.text}`, error.message);
                            // 将失败的链接信息也记录到结果中
                            this.results.push({
                                region: 'Europe',
                                standard: 'ERROR',
                                standardName: link.text || '未知链接',
                                dateOfPublication: '',
                                brief: `处理失败: ${error.message}`,
                                url: link.url || '',
                                error: true,
                                errorMessage: error.message,
                                timestamp: new Date().toISOString()
                            });
                        }
                    }
                    
                    // 关闭分类页面
                    await categoryPage.close();
                    
                    // 检查是否需要停止
                    if (shouldStop) {
                        console.log('检测到中断信号，跳出主循环');
                        break;
                    }
                    
                    // 更新进度
                    const progress = Math.round(((i + 1) / totalTargets) * 100);
                    if (progressCallback) {
                        progressCallback(progress, `已处理 ${i + 1}/${totalTargets} 个页面`, {
                            current: i + 1,
                            total: totalTargets,
                            category: categoryName,
                            status: 'success',
                            linksFound: links.length
                        });
                    }
                    
                } catch (error) {
                    console.error(`❌ 处理失败 ${categoryName}:`, error.message);
                    
                    // 更新进度
                    const progress = Math.round(((i + 1) / totalTargets) * 100);
                    if (progressCallback) {
                        progressCallback(progress, `处理失败 ${i + 1}/${totalTargets} 个页面`, {
                            current: i + 1,
                            total: totalTargets,
                            category: categoryName,
                            status: 'failed',
                            error: error.message
                        });
                    }
                }
            }
            
            // 检查是否因停止信号而中断
            if (!this.isRunning) {
                console.log(`\n⏹️ 抓取已停止! 共提取到 ${this.results.length} 条数据`);
            } else {
                console.log(`\n🎉 处理完成! 共提取到 ${this.results.length} 条数据`);
            }
            
            // 打印所有JSON数据
            if (this.results.length > 0) {
                console.log('\n📊 提取的JSON数据:');
                this.results.forEach((data, index) => {
                    console.log(`\n[${index + 1}] ${JSON.stringify(data, null, 2)}`);
                });
                
                // 导出JSON文件到data文件夹
                await this.saveJsonToFile();
            } else {
                console.log('\n⚠️ 没有提取到任何数据');
            }
            
            if (progressCallback) {
                progressCallback(100, '处理完成', {
                    current: totalTargets,
                    total: totalTargets,
                    category: '完成',
                    status: 'completed',
                    totalResults: this.results.length
                });
            }
            
            return this.results;
            
        } catch (error) {
            console.error('处理过程中发生错误:', error);
            throw error;
        }
    }

    // 提取JSON数据
    // 下载PDF并转换为文本
    async downloadPdfAndExtractText(pdfUrl) {
        return new Promise((resolve, reject) => {
            const protocol = pdfUrl.startsWith('https:') ? https : http;
            
            // 设置超时
            const timeout = setTimeout(() => {
                reject(new Error('PDF下载超时'));
            }, 30000); // 30秒超时
            
            const request = protocol.get(pdfUrl, (response) => {
                clearTimeout(timeout);
                
                // 处理重定向
                if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
                    console.log(`   🔄 PDF链接重定向到: ${response.headers.location}`);
                    this.downloadPdfAndExtractText(response.headers.location)
                        .then(resolve)
                        .catch(reject);
                    return;
                }
                
                if (response.statusCode !== 200) {
                    reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`));
                    return;
                }
                
                // 检查内容类型
                const contentType = response.headers['content-type'] || '';
                if (!contentType.includes('pdf') && !contentType.includes('application/octet-stream')) {
                    console.log(`   ⚠️ 警告: 内容类型可能不是PDF: ${contentType}`);
                }
                
                const chunks = [];
                let totalLength = 0;
                const maxSize = 50 * 1024 * 1024; // 50MB 最大文件大小
                
                response.on('data', (chunk) => {
                    totalLength += chunk.length;
                    if (totalLength > maxSize) {
                        reject(new Error('PDF文件过大，超过50MB限制'));
                        return;
                    }
                    chunks.push(chunk);
                });
                
                response.on('end', async () => {
                    try {
                        const buffer = Buffer.concat(chunks);
                        
                        // 检查是否是有效的PDF文件
                        if (buffer.length < 4 || buffer.toString('ascii', 0, 4) !== '%PDF') {
                            reject(new Error('下载的文件不是有效的PDF格式'));
                            return;
                        }
                        
                        const data = await pdf(buffer);
                        resolve(data.text);
                    } catch (error) {
                        reject(new Error(`PDF解析失败: ${error.message}`));
                    }
                });
                
                response.on('error', (error) => {
                    clearTimeout(timeout);
                    reject(new Error(`响应错误: ${error.message}`));
                });
            });
            
            request.on('error', (error) => {
                clearTimeout(timeout);
                reject(new Error(`请求错误: ${error.message}`));
            });
            
            // 设置请求超时
            request.setTimeout(30000, () => {
                request.destroy();
                reject(new Error('请求超时'));
            });
        });
    }

    // 从文本中提取日期
    extractDateFromText(text) {
        const datePattern = /\b\d{1,2}\.\d{1,2}\.\d{4}\b/g;
        const dateMatches = text.match(datePattern);
        return dateMatches && dateMatches.length > 0 ? dateMatches[0] : null;
    }

    async extractJsonData(detailPage, link) {
        try {
            console.log('链接为：', link.url)
            // 检查链接是否包含PDF
            if (link.url.includes('PDF')) {
                console.log(`   📄 检测到PDF链接，正在处理...`);
                
                let pdfText = null;
                let dateOfPublication = null;
                
                // 尝试直接下载PDF并转换为文本
                try {
                    console.log(`   📥 尝试下载PDF文件...`);
                    pdfText = await this.downloadPdfAndExtractText(link.url);
                    // console.log('PDF内容：', pdfText)
                    const rawDate = this.extractDateFromText(pdfText);
                    dateOfPublication = formatDateToEnglish(rawDate, false);
                    console.log('原始DATE内容：', rawDate);
                    console.log('转换后DATE内容：', dateOfPublication)
                    console.log(`   ✅ PDF下载成功，提取到文本长度: ${pdfText.length}`);
                } catch (pdfError) {
                    console.log(`   ⚠️ PDF下载失败: ${pdfError.message}`);
                    console.log(`   ℹ️ 跳过此PDF链接，继续处理下一个`);
                    return null;
                }
                
                // 如果成功提取到日期，构建返回数据
                if (dateOfPublication) {
                    // 获取standardName（链接名称）
                    const standardName = link.text;
                    
                    // 从链接名称中提取standard（查找Implementing后面的Decision格式）
                    let standard = null;
                    const implementingMatch = standardName.match(/Implementing.*?(Decision \(EU\) \d{4}\/\d+)/i);
                    if (implementingMatch) {
                        standard = implementingMatch[1];
                    } else {
                        // 如果没有找到Decision格式，尝试其他格式或使用链接名称的一部分
                        // 尝试匹配更完整的Commission标题
                        let commissionMatch = null;
                        
                        // 1. 优先匹配包含具体法规编号的完整标题
                        commissionMatch = standardName.match(/(Commission.*?(?:communication|regulation|directive).*?(?:Regulation|Directive).*?\d{4}\/\d+)/i);
                        
                        // 2. 如果没找到，匹配到OJ引用的完整标题
                        if (!commissionMatch) {
                            commissionMatch = standardName.match(/(Commission.*?(?:communication|regulation|directive).*?)(?:\s+-\s+OJ)/i);
                        }
                        
                        // 3. 如果还没找到，匹配到"in the framework"的部分
                        if (!commissionMatch) {
                            commissionMatch = standardName.match(/(Commission.*?(?:communication|regulation|directive).*?in\s+the\s+framework\s+of.*?)(?:\s+-|$)/i);
                        }
                        
                        // 4. 最后兜底，只匹配基本的Commission类型
                        if (!commissionMatch) {
                            commissionMatch = standardName.match(/(Commission.*?(?:communication|regulation|directive))/i);
                        }
                        if (commissionMatch) {
                            standard = commissionMatch[1];
                            console.log(`   🔍 匹配到的Commission标题: ${standard}`);
                        } else {
                            console.log(`   ⚠️ 未能匹配Commission格式，原标题: ${standardName}`);
                            // 如果都没找到，使用链接名称的前50个字符作为standard
                            standard = standardName.substring(0, 50) + (standardName.length > 50 ? '...' : '');
                        }
                    }
                    
                    console.log(`   📅 从PDF中提取到日期: ${dateOfPublication}`);
                    console.log(`   📋 提取到的standard: ${standard}`);
                    return {
                        region: 'Europe',
                        standard: standard,
                        standardName: standardName,
                        dateOfPublication: dateOfPublication,
                        brief: '',
                        url: link.url,
                    };
                } else {
                    console.log(`   ⚠️ 未能从PDF中提取到有效日期`);
                }
                
                return null;
            }
            
            // 非PDF链接的原有处理逻辑
            const jsonData = await detailPage.evaluate((linkData) => {
                // 获取standardName（链接名称）
                const standardName = linkData.text;
                
                // 从链接名称中提取standard（查找Implementing后面的Decision格式）
                let standard = null;
                const implementingMatch = standardName.match(/Implementing.*?(Decision \(EU\) \d{4}\/\d+)/i);
                if (implementingMatch) {
                    standard = implementingMatch[1];
                }
                
                // 获取url
                const url = linkData.url;
                
                // 查找dateOfPublication和提取文本内容用于AI总结
                let rawDate = null;
                let fullTextContent = '';
                const textTabContent = document.getElementById('textTabContent');
                if (textTabContent) {
                    const datePattern = /\b\d{1,2}\.\d{1,2}\.\d{4}\b/g;
                    const textContent = textTabContent.textContent || textTabContent.innerText || '';
                    fullTextContent = textContent; // 保存完整文本用于AI总结
                    const dateMatches = textContent.match(datePattern);
                    if (dateMatches && dateMatches.length > 0) {
                        rawDate = dateMatches[0];
                    }
                }
                
                // 只有当找到所有必要信息时才返回数据
                if (standardName && url && (standard || rawDate)) {
                // if (standardName && standard && url && rawDate) {
                    return {
                        region: 'Europe',
                        standard: standard,
                        standardName: standardName,
                        rawDate: rawDate, // 返回原始日期，在外部进行格式化
                        brief: '', // 将在外部调用AI总结后填充
                        fullTextContent: fullTextContent, // 临时字段，用于AI总结
                        url: url,
                        timestamp: new Date().toISOString()
                    };
                }
                
                return null;
            }, link);
            
            // 在Node.js环境中格式化日期
            if (jsonData && jsonData.rawDate) {
                jsonData.dateOfPublication = formatDateToEnglish(jsonData.rawDate, false);
                delete jsonData.rawDate; // 删除临时的原始日期字段
            }
            
            return jsonData;
            
        } catch (error) {
            console.error('提取JSON数据时发生错误:', error.message);
            return null;
        }
    }

    // 使用信号运行可中断任务
    async runWithSignal(signal, headless = true, progressCallback = null) {
        try {
            // 重置文件保存标志，允许新的抓取任务保存文件
            this.fileSaved = false;
            
            // 检查信号是否已经被中止
            if (signal && signal.aborted) {
                console.log('任务在启动前已被取消');
                return { success: false, message: '任务已被用户取消', results: [] };
            }

            console.log('使用可中断模式启动EU Commission抓取任务...');
            
            // 初始化浏览器（传入信号）
            const initSuccess = await this.init(headless, signal);
            if (!initSuccess) {
                return { success: false, message: '浏览器初始化失败', results: [] };
            }

            // 再次检查信号
            if (signal && signal.aborted) {
                console.log('任务在初始化后被取消');
                await this.close();
                return { success: false, message: '任务已被用户取消', results: [] };
            }

            // 开始抓取
            this.isRunning = true;
            await this.scrapeTargetPages(progressCallback || ((progress, message) => {
                console.log(`进度: ${progress}% - ${message}`);
            }));

            // 检查是否因为信号中断而停止
            if (signal && signal.aborted) {
                console.log('抓取过程中检测到中断信号');
                // 保存现有结果
                if (this.results && this.results.length > 0) {
                    console.log(`保存中断前的结果 (${this.results.length} 条记录)`);
                    await this.saveJsonToFile();
                }
                await this.close();
                return { 
                    success: false, 
                    message: '任务已被用户中断，现有数据已保存', 
                    results: this.results 
                };
            }

            // 正常完成
             console.log(`抓取完成，共获取 ${this.results.length} 条记录`);
             await this.saveJsonToFile();
             await this.close();
            
            return { 
                success: true, 
                message: `抓取完成，共获取 ${this.results.length} 条记录`, 
                results: this.results 
            };

        } catch (error) {
            console.error('抓取过程中发生错误:', error);
            
            // 即使出错也要保存现有结果
            if (this.results && this.results.length > 0) {
                console.log(`保存错误前的结果 (${this.results.length} 条记录)`);
                await this.saveJsonToFile();
            }
            
            await this.close();
            
            // 如果是用户取消的错误，返回特殊消息
            if (error.message.includes('取消') || (signal && signal.aborted)) {
                return { 
                    success: false, 
                    message: '任务已被用户取消，现有数据已保存', 
                    results: this.results 
                };
            }
            
            return { 
                success: false, 
                message: `抓取失败: ${error.message}`, 
                results: this.results 
            };
        }
    }

    // 停止抓取
    async stop() {
        console.log('正在停止EU Commission抓取...');
        this.isRunning = false;
        
        // 如果有结果，保存到JSON文件
        if (this.results && this.results.length > 0) {
            console.log(`保存现有结果 (${this.results.length} 条记录)`);
            await this.saveJsonToFile();
        }
    }

    // 关闭浏览器
    async close() {
        this.isRunning = false;
        if (this.context) {
            await this.context.close();
            this.context = null;
        }
        if (this.browser) {
            await this.browser.close();
            this.browser = null;
            this.page = null;
            console.log('EU Commission浏览器已关闭');
        }
    }

    // 导出结果到JSON
    exportToJSON() {
        const exportData = {
            exportTime: new Date().toISOString(),
            totalRecords: this.results.length,
            results: this.results
        };
        
        return JSON.stringify(exportData, null, 2);
    }

    // 获取结果统计
    getStats() {
        return {
            total: this.results.length,
            results: this.results
        };
    }

    // 保存JSON文件到data文件夹
    async saveJsonToFile() {
        try {
            // 检查是否已经保存过文件，防止重复保存
            if (this.fileSaved) {
                console.log('文件已保存，跳过重复保存');
                return;
            }
            
            // 清理旧数据文件
            try {
                cleanupModuleData('eucommission');
            } catch (cleanupError) {
                console.log(`清理旧数据时出现警告: ${cleanupError.message}`);
            }
            
            // 使用统一的数据路径管理
            const dataDir = pathUtils.getDataPath('eucommission');
            pathUtils.ensureDirectoryExists(dataDir);
            
            // 生成文件名（包含时间戳）
            const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
            const filename = `eu-commission-standards-${timestamp}.json`;
            const filepath = path.join(dataDir, filename);
            
            // 准备导出数据
            const exportData = {
                exportTime: new Date().toISOString(),
                totalRecords: this.results.length,
                results: this.results
            };
            
            // 写入文件
            fs.writeFileSync(filepath, JSON.stringify(exportData, null, 2), 'utf8');
            
            // 标记文件已保存
            this.fileSaved = true;
            
            console.log(`\n💾 JSON文件已保存到: ${filepath}`);
            console.log(`📊 共保存 ${this.results.length} 条记录`);
            
        } catch (error) {
            console.error('保存JSON文件失败:', error.message);
        }
    }
}

// 导出类和函数
// 主执行函数
async function main(isHeadless = true) {
    try {
        console.log('开始执行EU Commission标准数据获取...', 'info');
        
        const scraper = new EuCommissionScraper();
        await scraper.init(isHeadless);
        
        const results = await scraper.scrapeTargetPages((progress, message) => {
            console.log(`进度: ${progress}% - ${message}`);
        });
        
        const jsonData = scraper.exportToJSON();
        await scraper.close();
        
        console.log('EU Commission标准数据获取完成', 'success');
        return {
            success: true,
            data: results,
            json: jsonData,
            stats: scraper.getStats()
        };
        
    } catch (error) {
        console.error(`执行失败: ${error.message}`, 'error');
        return {
            success: false,
            error: error.message,
            data: []
        };
    }
}

// 模块导出
module.exports = {
    EuCommissionScraper,
    TARGET_LINKS,
    BASE_URL,
    MAIN_URL
};

// 如果作为主模块运行
if (require.main === module) {
    // 从命令行参数获取headless设置
    const isHeadless = process.argv[2] === 'true' || process.argv[2] === true;
    console.log('EU Commission Core: 从命令行参数获取headless设置:', isHeadless);
    
    main(isHeadless).then(result => {
        console.log('EU Commission Core: 执行结果:', result);
        process.exit(0);
    }).catch(err => {
        console.error('EU Commission Core: 执行失败:', err);
        process.exit(1);
    });
} else {
    // 创建一个支持中断信号的包装函数
    async function runWithSignal(signal, headless = true, progressCallback = null) {
        const scraper = new EuCommissionScraper();
        return await scraper.runWithSignal(signal, headless, progressCallback);
    }
    
    // 正确的模块导出方式
    module.exports = main; // 默认导出main函数
    module.exports.main = main;
    module.exports.runWithSignal = runWithSignal; // 添加中断信号支持
    module.exports.EuCommissionScraper = EuCommissionScraper;
    module.exports.TARGET_LINKS = TARGET_LINKS;
    module.exports.BASE_URL = BASE_URL;
    module.exports.MAIN_URL = MAIN_URL;
}